﻿//////////////////////////////////////////////////////////////////
//
// Bruno\Main.cs
//
// Copyright (c) 2012 Dan Pike.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation files
// (the "Software"), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Threading;
using System.Windows.Forms;
using Utilities;

using HtmlDocument = HtmlAgilityPack.HtmlDocument;
using System.IO;
using System.Text;
using System.Windows;
using System.Text.RegularExpressions;

namespace Bruno
{
   public partial class Main : Form
   {
      private static Logger logger_;
      public ListView CodeListView;
      public string UserName;
      public string Password;
      public string LoginServer;

      private Queue<Code> codeMonitor_;
      private Dictionary<Code, bool> codeDisplay_;

      private const int NUMBER_OF_DAYS = 6;
      private Thread threadCheck_;
      private EventWaitHandle stopThreads_;
      private List<Thread> poolThreads_;

      private Regex regex_;

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public Main()
      {
         InitializeComponent();

         CodeListView = lvCode;
         logger_ = Program.Logger;
         codeMonitor_ = new Queue<Code>();
         codeDisplay_ = new Dictionary<Code, bool>();
         poolThreads_ = new List<Thread>();
         regex_ = new Regex(@"^(?<year>\d{4})(?<month>\d{2})(?<day>\d{2})(?:\.(?<page>\d+))?$");

         SetupGrid();
         Width = 1100;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      private void SetupGrid()
      {
         lvCode.Dock = DockStyle.Fill;
         lvCode.View = View.Details;
         lvCode.FullRowSelect = true;
         lvCode.GridLines = true;

         AddColumn(100, "Code");
         AddColumn(200, "Status");
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="width"></param>
      /// <param name="format"></param>
      /// <param name="args"></param>
      private void AddColumn(int width, string format, params object[] args)
      {
         ColumnHeader column = new ColumnHeader();
         column.Text = string.Format(format, args);
         column.Width = width;
         if (0 < args.Length)
         {
            column.Name = ((DateTime)args[0]).ToString("yyyyMMdd");
            column.Tag = args[0];
            column.TextAlign = HorizontalAlignment.Center;
            lvCode.Columns.Insert(2, column);
         }
         else
         {
            lvCode.Columns.Add(column);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="sender"></param>
      /// <param name="e"></param>
      private void Main_Load(object sender, EventArgs e)
      {
         try
         {
            foreach (Config.Code configCode in Program.Settings.Codes.Value)
            {
               Code code = new Code();
               code.Name = configCode.Name;
               code.Settings = configCode;
               code.Item = lvCode.Items.Add(code.Name);
               codeMonitor_.Enqueue(code);
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="sender"></param>
      /// <param name="e"></param>
      private void Main_FormClosing(object sender, FormClosingEventArgs e)
      {
         try
         {
            logger_.DebugFormat("signalling threads to shut down");
            stopThreads_.Set();
            if (null != threadCheck_)
            {
               threadCheck_.Join();
               logger_.DebugFormat("checker thread has joined");
            }
            poolThreads_.ForEach(poolThread =>
               {
                  poolThread.Join();
                  logger_.DebugFormat("{0} has joined", poolThread.Name);
               });

            Program.Settings.WindowState.Value = WindowState;
            if (FormWindowState.Normal == WindowState)
            {
               Program.Settings.Window.Value = new Rectangle(Left, Top, Width, Height);
            }
            else
            {
               Program.Settings.Window.Value = new Rectangle(0, 0, 0, 0);
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="sender"></param>
      /// <param name="e"></param>
      private void Main_Shown(object sender, EventArgs e)
      {
         try
         {
            Rectangle client = Program.Settings.Window.Value;
            if ((0 < client.Width) && (0 < client.Height))
            {
               WindowState = Program.Settings.WindowState;
               if (FormWindowState.Normal == WindowState)
               {
                  Left = client.Left;
                  Top = client.Top;
                  Width = client.Width;
                  Height = client.Height;
               }
            }

            DialogResult result = UserDetails.GetUserId(this, "Enter login details",
               Program.AppConfig, false, out UserName, out Password, out LoginServer);
            if (DialogResult.Yes != result)
            {
               Close();
               return;
            }

            logger_.DebugFormat("creating checker thread");
            stopThreads_ = new EventWaitHandle(false, EventResetMode.ManualReset);
            threadCheck_ = new Thread(new ThreadStart(() => { threadChecker(); }));
            threadCheck_.Name = "Checker";
            threadCheck_.Start();
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="sender"></param>
      /// <param name="e"></param>
      private void exitToolStripMenuItem_Click(object sender, EventArgs e)
      {
         Close();
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      private void threadChecker()
      {
         try
         {
            logger_.DebugFormat("checker thread is starting");
            Queue<string> archiveList = new Queue<string>();
            int maxThreads = -1;
            bool done = false;
            DateTime lastArchiveCheck = DateTime.MinValue;
            while (!done)
            {
               if (false != (done = stopThreads_.WaitOne(3000)))
               {
                  logger_.DebugFormat("checker thread has detected shutdown request");
                  break;
               }

               // Use consistent date values on each iteration
               DateTime now = DateTime.Now;
               DateTime today = now.Date;
               TimeSpan timeNow = now.TimeOfDay;

               // Check the pool threads
               maxThreads = adjustPoolThreadCount(timeNow, maxThreads);

               if (lastArchiveCheck + new TimeSpan(1, 0, 0) <= now)
               {
                  // Archive old files
                  if (checkArchiveFiles(today - new TimeSpan(6, 0, 0, 0), ref archiveList))
                  {
                     lastArchiveCheck = now;
                  }
               }

               // Create the column, if this is a weekday and we do not have it
               //if ((DayOfWeek.Saturday != today.DayOfWeek) && (DayOfWeek.Sunday != today.DayOfWeek))
               //{
               //   string key = today.ToString("yyyyMMdd");
               //   SafeAsyncInvoker.BeginInvoke(lvCode, () =>
               //   {
               //      if (!lvCode.Columns.ContainsKey(key))
               //      {
               //         AddColumn(100, "{0:ddd, dd MMM}", today);
               //      }
               //   });
               //}

               // Move old files to the archive folder

            }
            logger_.DebugFormat("checker thread is stopping");
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Adjust the thread pool according to the time of day
      /// </summary>
      /// <param name="timeNow"></param>
      /// <param name="maxThreads"></param>
      /// <returns></returns>
      private int adjustPoolThreadCount(TimeSpan timeNow, int maxThreads)
      {
         try
         {
            // Drop any threads that have died
            int threadIndex = 0;
            poolThreads_.ToList().ForEach(thread =>
               {
                  if (!thread.IsAlive)
                  {
                     logger_.DebugFormat("dropping dead thread #{0}", threadIndex);
                     poolThreads_.Remove(thread);
                  }
                  ++threadIndex;
               });

            // And start any news ones that we need
            int newMaxThreads = ((7.0 > timeNow.TotalHours) || (18.0 < timeNow.TotalHours)) ? 8 : 1;
            if (maxThreads != newMaxThreads)
            {
               logger_.DebugFormat("changing thread pool to size {0}", newMaxThreads);
               maxThreads = newMaxThreads;

               while (maxThreads > poolThreads_.Count)
               {
                  int poolIndex = poolThreads_.Count;
                  Thread poolThread = new Thread(new ThreadStart(() => { poolWatch(poolIndex); }));
                  poolThread.Name = string.Format("pool thread #{0}", poolIndex);
                  poolThreads_.Add(poolThread);
                  poolThread.Start();
               }
               if (maxThreads < poolThreads_.Count)
               {
                  // Any active threads will cancel themselves automatically when they
                  // detect that the pool has changed size
                  poolThreads_.RemoveRange(maxThreads, poolThreads_.Count - maxThreads);
               }
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
         return maxThreads;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Check the next folder for files that we can archive
      /// </summary>
      /// <param name="archiveList"></param>
      /// <returns></returns>
      private bool checkArchiveFiles(DateTime oldestFile, ref Queue<string> archiveList)
      {
         try
         {
            if (0 == archiveList.Count)
            {
               // Take another snapshot of the root folder
               foreach(string folder in Directory.EnumerateDirectories(Program.Settings.Folder.Value))
               {
                  archiveList.Enqueue(folder);
               }
            }
            if (0 < archiveList.Count)
            {
               // Check the next folder in the list
               string archiveFolder = Program.Settings.Archive.Value;
               if (!Directory.Exists(archiveFolder))
               {
                  logger_.DebugFormat("creating archive folder '{0}'", archiveFolder);
                  Directory.CreateDirectory(archiveFolder);
               }
               string folder = archiveList.Dequeue();
               logger_.DebugFormat("checking archive '{0}'", folder);
               foreach (string pathname in Directory.EnumerateFiles(folder))
               {
                  string filename = Path.GetFileNameWithoutExtension(pathname);

                  // If the date is older than one week, we archive it
                  Match match = regex_.Match(filename);
                  if (match.Success)
                  {
                     // Parse the filename
                     DateTime fileDate = new DateTime(
                        int.Parse(match.Groups["year"].Value),
                        int.Parse(match.Groups["month"].Value),
                        int.Parse(match.Groups["day"].Value));

                     // Is the file too old?
                     if (oldestFile > fileDate)
                     {
                        string archiveDate = fileDate.ToString("yyyyMM");
                        string shareFolder = Path.Combine(archiveFolder, archiveDate, Path.GetFileName(folder));
                        if (!Directory.Exists(shareFolder))
                        {
                           logger_.DebugFormat("creating folder '{0}' for share archive", shareFolder);
                           Directory.CreateDirectory(shareFolder);
                        }

                        string archivePath = string.Concat(Path.Combine(shareFolder, filename), ".html");
                        logger_.DebugFormat("moving '{0}' to '{1}'", pathname, archivePath);
                        try
                        {
                           File.Move(pathname, archivePath);
                        }
                        catch (System.Exception ex)
                        {
                           logger_.Exception("moving '{0}' to '{1}'", ex, pathname, archivePath);
                        }
                     }
                  }
               }
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
         // If that was the last one in our archive, then we wait for five mins before trying again
         return (0 == archiveList.Count);
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="poolIndex"></param>
      private void poolWatch(int poolIndex)
      {
         try
         {
            logger_.DebugFormat("pool thread {0} is starting", poolIndex);

            Pager pager = new Pager();
            pager.UserName = UserName;
            pager.Password = Password;
            pager.LoginServer = LoginServer;
            pager.Login();

            bool done = false, didWork = false;
            while (!done)
            {
               int delay = didWork ? 0 : 3000;
               if (false != (done = stopThreads_.WaitOne(100)))
               {
                  logger_.DebugFormat("pool thread {0} has detected shutdown request", poolIndex);
                  break;
               }
               didWork = false;

               int poolSize = poolThreads_.Count;
               if (poolSize <= poolIndex)
               {
                  logger_.DebugFormat("pool thread {0} is no longer required (pool size is now {1})",
                     poolIndex, poolSize);
                  done = true;
                  break;
               }

               // Get a code to work on
               Code code = null;
               try
               {
                  lock (codeMonitor_)
                  {
                     if (0 < codeMonitor_.Count)
                     {
                        code = codeMonitor_.Dequeue();
                     }
                  }
                  if (null != code)
                  {
                     lock (codeDisplay_)
                     {
                        codeDisplay_[code] = true;
                     }

                     DateTime codeDate = CheckDate.Date;
                     for (int dayIndex = 0; dayIndex < 1; ++dayIndex)
                     {
                        if (PoolProcessCode(pager, code, codeDate))
                        {
                           didWork = true;
                        }
                        do
                        {
                           codeDate -= new TimeSpan(1, 0, 0, 0);
                        } while ((DayOfWeek.Saturday == codeDate.DayOfWeek) || (DayOfWeek.Sunday == codeDate.DayOfWeek));
                     }

                     lock (codeDisplay_)
                     {
                        codeDisplay_[code] = false;
                     }
                  }
               }
               finally
               {
                  // Be sure to put this code back on the list!
                  if (null != code)
                  {
                     lock (codeMonitor_)
                     {
                        codeMonitor_.Enqueue(code);
                     }
                  }
               }
            }

            logger_.DebugFormat("pool thread {0} is exiting", poolIndex);
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Process the next code in the collection, check for holes and request
      /// data until we get it all
      /// </summary>
      /// <param name="code"></param>
      private bool PoolProcessCode(Pager pager, Code code, DateTime codeDate)
      {
         bool didWork = false;
         try
         {
            DateTime now = CheckDate;
            DateTime today = now.Date;
            TimeSpan fivePm = new TimeSpan(17, 0, 0);
            TimeSpan timeNow = now.TimeOfDay;

            // Check each page in turn until we get to the end of the list
            int pageIndex = 0;

            // Have we had at least some of this trading day?
            DateTime firstMarketCheck = codeDate + new TimeSpan(7, 0, 0);
            DateTime lastMarketCheck = codeDate + new TimeSpan(22, 0, 0);
            if (now >= codeDate)
            {
               HtmlDocument doc = null;
               //logger_.DebugFormat("processing code '{0}'", code.Name);

               // Check the intraday chart page first
               if (fivePm < timeNow)
               {
                  if (PoolCheckChartPage(pager, code, codeDate))
                  {
                     didWork = true;
                  }
               }

               while (!stopThreads_.WaitOne(0))
               {
                  // Have we already downloaded this page?
                  string thisFilename = code.GetFilename(codeDate, pageIndex);
                  if (File.Exists(thisFilename))
                  {
                     // Read the existing file, so we can check to see if it has a "next" link
                     doc = new HtmlDocument();
                     doc.Load(thisFilename);
                     if (null == doc.DocumentNode)
                     {
                        // Failed to parse it, so overwrite it
                        doc = null;
                     }
                  }

                  // If this is the last file in the sequence, and the file is old enough
                  // and the latest entry in the file is far from the end of day,
                  // then we snap the page again
                  bool lastFile = (null != doc) && string.IsNullOrEmpty(Code.GetNextUrl(doc));
                  if (lastFile)
                  {
                     if (code.MinAge < (now - File.GetLastWriteTime(thisFilename)).TotalMinutes)
                     {
                        PoolCheckNewestEntry(ref doc, lastMarketCheck);
                     }
                  }

                  if (null == doc)
                  {
                     // Download a (possibly replacement) copy of the page
                     doc = pager.Download(stopThreads_, code, codeDate, pageIndex);
                     if (null != doc)
                     {
                        // Is this the last file in the sequence?
                        lastFile = string.IsNullOrEmpty(Code.GetNextUrl(doc));

                        // Save it to disk
                        string directoryName = Path.GetDirectoryName(thisFilename);
                        if (!Directory.Exists(directoryName))
                        {
                           logger_.DebugFormat("creating folder '{0}'", directoryName);
                           Directory.CreateDirectory(directoryName);
                        }
                        doc.Save(thisFilename);
                        didWork = true;
                     }
                  }

                  // Update the display with the name of the latest file that we worked on
                  StringBuilder display = new StringBuilder();
                  display.Append(Path.GetFileNameWithoutExtension(thisFilename).Replace(".", ", #"));
                  HtmlNodeCollection nodes = Code.GetAllNodes(doc);
                  if ((null != nodes) && (0 < nodes.Count))
                  {
                     DateTime lastEntry;
                     if (Code.ParseEntryTime(nodes[nodes.Count - 1], out lastEntry))
                     {
                        display.Append(", ");
                        display.Append(lastEntry.ToString("HH:mm"));
                     }
                  }
                  if (lastFile)
                  {
                     display.Append(" (completed)");
                  }
                  code.CurrentFilename = display.ToString();

                  // If this page does not have a "Next" link, then we have reached
                  // the end of the sequence
                  if (lastFile)
                  {
                     // No more pages
                     break;
                  }

                  // Test out the next page in the sequence
                  doc = null;
                  ++pageIndex;
               }

               // logger_.DebugFormat("finished processing code '{0}'", code.Name);
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
         return didWork;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Download the intra-day chart page
      /// </summary>
      /// <param name="pager"></param>
      /// <param name="code"></param>
      /// <param name="codeDate"></param>
      private bool PoolCheckChartPage(Pager pager, Code code, DateTime codeDate)
      {
         try
         {
            string thisFilename = code.GetIntradayFilename(codeDate);
            if (File.Exists(thisFilename))
            {
               // We already downloaded it
               return false;
            }
            code.CurrentFilename = Path.GetFileNameWithoutExtension(thisFilename) + " - intraday";

            HtmlDocument doc = pager.DownloadIntraday(stopThreads_, code, codeDate);
            if (null != doc)
            {
               // Clear the page
               HtmlNode rootNode = doc.DocumentNode;
               rootNode.SelectNodes("//img").ToList().ForEach(node => node.Remove());
               rootNode.SelectNodes("//link").ToList().ForEach(node => node.Remove());
               rootNode.SelectNodes("//meta").ToList().ForEach(node => node.Remove());
               rootNode.SelectNodes("//form").ToList().ForEach(node => node.Remove());
               rootNode.SelectNodes("//ul").ToList().ForEach(node => node.Remove());

               rootNode.SelectNodes("//script").ToList().ForEach(node =>
               {
                  // Keep only the script that has the ticks in it
                  if (!node.InnerText.Contains("intradayDataString"))
                  {
                     node.Remove();
                  }
               });

               // Save it to disk
               string directoryName = Path.GetDirectoryName(thisFilename);
               if (!Directory.Exists(directoryName))
               {
                  logger_.DebugFormat("creating folder '{0}'", directoryName);
                  Directory.CreateDirectory(directoryName);
               }
               doc.Save(thisFilename);
               return true;
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
         }
         return false;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// Reset the HtmlDocument if it's not full and the newest entry is too
      /// old and we should download the page again to see if there's any more
      /// </summary>
      /// <param name="doc"></param>
      /// <param name="lastMarketCheck"></param>
      /// <returns></returns>
      private DateTime PoolCheckNewestEntry(ref HtmlDocument doc, DateTime lastMarketCheck)
      {
         DateTime lastEntry = DateTime.MinValue;
         HtmlNodeCollection nodes = Code.GetAllNodes(doc);
         if ((null == nodes) || (0 == nodes.Count))
         {
            // There are no entries on this page, so snap it again
            doc = null;
         }
         else
         {
            if (Code.ParseEntryTime(nodes[nodes.Count - 1], out lastEntry) &&
               (lastMarketCheck > lastEntry))
            {
               // Final entry on this page is not close enough to the end-of-day
               // we should snap again
               doc = null;
            }
         }
         return lastEntry;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="sender"></param>
      /// <param name="e"></param>
      private void guiTimer_Tick(object sender, EventArgs e)
      {
         // Get the next code to update in the display
         List<KeyValuePair<Code, bool>> codes;
         lock (codeDisplay_)
         {
            codes = codeDisplay_.ToList();
         }
         codes.ForEach(kvp =>
            {
               Code code = kvp.Key;
               string thisFilename = code.CurrentFilename;
               if (null != thisFilename)
               {
                  code.Item.SubItems[1].Text = code.CurrentFilename;
                  if (!kvp.Value)
                  {
                     // We've completed the first run, so we don't check again for an hour
                     code.MinAge = 60.0;
                     lock (codeDisplay_)
                     {
                        codeDisplay_.Remove(code);
                     }
                  }
               }
            });
      }

      private DateTime CheckDate
      {
         get
         {
            return DateTime.Now; // new DateTime(2012, 4, 11, 19, 0, 0);
         }
      }
   }
}
